{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5d244f26",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "array([ 0.00686026,  1.421801  ,  0.6948606 ,  0.48357785, -0.0079426 ,\n",
       "       -0.15739633,  0.        ,  0.        ], dtype=float32)"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import gym\n",
    "\n",
    "\n",
    "#定义环境\n",
    "class MyWrapper(gym.Wrapper):\n",
    "\n",
    "    def __init__(self):\n",
    "        env = gym.make('LunarLander-v2',\n",
    "                       continuous=False,\n",
    "                       render_mode='rgb_array')\n",
    "        super().__init__(env)\n",
    "        self.env = env\n",
    "        self.step_n = 0\n",
    "\n",
    "    def reset(self):\n",
    "        state, _ = self.env.reset()\n",
    "        self.step_n = 0\n",
    "        return state\n",
    "\n",
    "    def step(self, action):\n",
    "        state, reward, done, _, info = self.env.step(action)\n",
    "        self.step_n += 1\n",
    "        if self.step_n >= 400:\n",
    "            done = True\n",
    "        return state, reward, done, info\n",
    "\n",
    "\n",
    "env = MyWrapper()\n",
    "\n",
    "env.reset()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "b77c66d3",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAZqElEQVR4nO3dfXBU5d038O++Z0N2Ny+72U1usrxYLWKQYoAk8IxWWYiAVtBOrYNTprY6SGilRmdIO+rwzDjxsXN7P2Ip9WWKnce70oEOVTFSaaJBp5GQkABJIFgEEjW7gUB2E8j7/p4/cueURSQvJLmyyfczcznmXNee8zsnOV+uPWdfdCIiICJSQK+6ACKavBhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkjLIA2rp1K6ZPn46YmBhkZmairKxMVSlEpIiSAPrLX/6CJ598Es899xwOHTqEuXPnIicnB01NTSrKISJFdCrejJqZmYkFCxbgd7/7HQAgHA4jLS0Nv/jFL7Bp06axLoeIFDGO9Qa7urpQUVGB/Px8bZler4fP50NpaelVH9PZ2YnOzk7t53A4jPPnzyMpKQk6nW7UayaioRERtLa2IjU1FXr9tz/RGvMAOnfuHHp7e+F2uyOWu91uHD9+/KqPKSgowObNm8eiPCIaQQ0NDZg6deq39o95AA1Hfn4+nnzySe3nYDAIr9ersCIaDFfiTfjBnf8HsWYn3v3kSZz58iDuvn0zbvHeC5NxyqDX09Hdgv/3wUMIhQJY9r9+je9OvRsVJ/6E/Qd/B5HwKO4BXS+bzXbN/jEPIKfTCYPBgEAgELE8EAjA4/Fc9TEWiwUWi2UsyqMRotPpceus1Uh2zEblv/4bXzZW9S2HDq3dX8PYawFw5dNn3b+XXPbUuqunDdAJunsu4mjd3zAtJRPfu+khHDv1dzSdrRuDvaHhGugSyZgHkNlsRkZGBoqKirBq1SoAfdd0ioqKsGHDhrEuh0ZJrDURKcmzEbzUgPKat9Db2w0AKCn7vzAYTFeMlsv+G/l/gEBE0N4RBAB8GajEsVOFyJz9c9w6axWKm/8T4XDPaO4KjSZRYMeOHWKxWOTNN9+U2tpaeeyxxyQ+Pl78fv+gHh8MBgV9f6Vs47ilJM+Rm2/KEZ1OP6LrTbB7ZcVd/1vS/iNjxNfNNrItGAxe81xWEkAiIq+88op4vV4xm82ycOFC+eyzzwb9WAYQm8loVV4D28BtoABS8jqg6xUKheBwOFSXQUQDCAaDsNvt39rP94IRkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhImSEH0P79+3HvvfciNTUVOp0Of/vb3yL6RQTPPvssUlJSYLVa4fP58Pnnn0eMOX/+PNasWQO73Y74+Hj87Gc/Q1tb23XtCBFFnyEH0MWLFzF37lxs3br1qv0vvvgitmzZgj/84Q84cOAApkyZgpycHHR0dGhj1qxZg5qaGuzbtw979uzB/v378dhjjw1/L4goOsl1ACC7d+/Wfg6Hw+LxeOS3v/2ttqylpUUsFou8/fbbIiJSW1srAOTgwYPamA8++EB0Op189dVXg9puMBgUAGxsbOO8BYPBa57LI3oN6NSpU/D7/fD5fNoyh8OBzMxMlJaWAgBKS0sRHx+P+fPna2N8Ph/0ej0OHDhw1fV2dnYiFApFNCKKfiMaQH6/HwDgdrsjlrvdbq3P7/cjOTk5ot9oNCIxMVEbc6WCggI4HA6tpaWljWTZRKRIVNwFy8/PRzAY1FpDQ4PqkohoBIxoAHk8HgBAIBCIWB4IBLQ+j8eDpqamiP6enh6cP39eG3Mli8UCu90e0Ygo+o1oAM2YMQMejwdFRUXaslAohAMHDiA7OxsAkJ2djZaWFlRUVGhjiouLEQ6HkZmZOZLlENF4N4SbXiIi0traKpWVlVJZWSkA5KWXXpLKyko5c+aMiIi88MILEh8fL++8844cOXJE7rvvPpkxY4a0t7dr67j77rtl3rx5cuDAAfn000/lxhtvlIceemjQNfAuGBtbdLSB7oINOYA++uijq25o7dq1ItJ3K/6ZZ54Rt9stFotFlixZInV1dRHraG5uloceekji4uLEbrfLT3/6U2ltbWUAsbFNsDZQAOlERBBlQqEQHA6H6jKIaADBYPCa12yj4i4YEU1MDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISJkhBVBBQQEWLFgAm82G5ORkrFq1CnV1dRFjOjo6kJubi6SkJMTFxeGBBx5AIBCIGFNfX4+VK1ciNjYWycnJePrpp9HT03P9e0NE0UWGICcnR7Zv3y7V1dVSVVUlK1asEK/XK21tbdqYdevWSVpamhQVFUl5eblkZWXJokWLtP6enh5JT08Xn88nlZWVUlhYKE6nU/Lz8wddRzAYFABsbGzjvAWDwWuey0MKoCs1NTUJACkpKRERkZaWFjGZTLJz505tzLFjxwSAlJaWiohIYWGh6PV68fv92pht27aJ3W6Xzs7OQW2XAcTGFh1toAC6rmtAwWAQAJCYmAgAqKioQHd3N3w+nzZm1qxZ8Hq9KC0tBQCUlpZizpw5cLvd2picnByEQiHU1NRcdTudnZ0IhUIRjYii37ADKBwOY+PGjVi8eDHS09MBAH6/H2azGfHx8RFj3W43/H6/Nuby8Onv7++7moKCAjgcDq2lpaUNt2wiGkeGHUC5ubmorq7Gjh07RrKeq8rPz0cwGNRaQ0PDqG+TiEafcTgP2rBhA/bs2YP9+/dj6tSp2nKPx4Ouri60tLREzIICgQA8Ho82pqysLGJ9/XfJ+sdcyWKxwGKxDKdUIhrPhnLRORwOS25urqSmpsqJEye+0d9/EXrXrl3asuPHjwvwzYvQgUBAG/Pqq6+K3W6Xjo6OQdXBi9BsbNHRRvQu2OOPPy4Oh0M+/vhjaWxs1NqlS5e0MevWrROv1yvFxcVSXl4u2dnZkp2drfX334ZftmyZVFVVyd69e8XlcvE2PBvbBGwjGkDftpHt27drY9rb22X9+vWSkJAgsbGxsnr1amlsbIxYz+nTp2X58uVitVrF6XRKXl6edHd3M4DY2CZYGyiAdP8TLFElFArB4XCoLoOIBhAMBmG327+1n+8FIyJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlBnW5wHR5KPTAfffDxgMQGUlEAwC584B4bDqygbmdgP33gucPNnXLlwAWltVV0UAA4gGSacD7rsPmDUL6O4GQiHg9Gngyy/7Asnv7zu5OzqAzk7V1UZyuYBHHwVE+mr/+mugsRGorga++KKvnTsHtLcDvb2qq51cGEA0JHo9YLH0ndQuF7BgAbBqVV/wXLgAHDsG/Nd/9QXSeKLT9c3ejEbghhv62uLFfaEUCgHnzwN79gBvvRUds7qJggFEQ9Y/kwiFgKYmoLYWqK8HDh/uO5Gv+B7KcUOkr1261PcU7PPP++o+cqRvJtfYyPAZa5M2gHQ6HfR6PfR6PQwGA2w2m/atG3q9HkePHkVHR4fqMseV8+c9KCz0o7y8L2ROnOib+bS3q67s2vR6G44eFVRWtqGuDjhzpi9sOjr6gnS80uv1MBqN6O3tRe8EfW44YQPIaDTCarVqLTExEW63G8nJyXC73XC73fB4PPB4PEhOTobNZkNsbCymTJkCnU6H/fv34+WXX8bHH3/MIAKg0xlQWZmDP/3pT6pLGTKT6SZs2dKDw4cPqy7lmvR6PeLj4+H1epGRkYHMzEzMmTMHp0+fRkVFBSorK1FXV4fm5ma0j/fUH6SoDqCZM2ciKSkJiYmJcLlcSElJQWpqKlJSUuB0OpGQkICEhATEx8cjJiZGm+0YDAbodLprrjsnJwd33HEHSkpK8PLLL6OkpGTC/NJpfLBYLEhMTMRNN92E+fPnIzs7G7Nnz0ZaWhqsViv0ej10Oh2ysrLwox/9CN3d3bhw4QJOnjyJI0eOoKysDDU1NTh58iTa2trQ1dWlepeGLKo/kvXkyZNwuVyIiYmB0WgcMFSGq729XQuiyTojMhgMePjhh6NyBpSRkYGeHvUzoJiYGKSkpOCWW25BVlYW5s+fj9mzZ8PpdCImJmbIf78igra2Npw9exY1NTWorKxERUUFamtr4ff70dbWNkp7MngDfSRrVM+AnE4nbDbbqG/HarUiJycH3//+91FSUoItW7bgo48+4oyIvpXBYEBcXBymTZuGuXPnIisrCxkZGZg5cyYSEhIGNQsfiE6ng81mg81mw4wZM3DPPfegt7cXoVAI9fX1qK6uRllZGY4cOYITJ06gpaVl3P3NRnUAjSWdToeYmBjk5OTg9ttvjwiiyTgjokgGgwFJSUmYOXMm5s+fj6ysLMydOxdTp06FzWaDwWAY1e33h5nRaERiYiISExPxve99D2vWrEF7ezsuXLiAuro6HD58GAcPHkRNTQ3q6+sRDAah8kkQA2gYLp8R7d+/H1u2bEFxcfG4+9eFRpfJZMKsWbPwgx/8AIsWLcLNN9+MlJQU7Vt8R+uSwFDodDrExsYiNjYWqampuPPOOyEiuHjxIr788ku89tpreOONN9Q9XRv0l3GNI/3fCzbQdw6NlUuXLsnf//53WbFihVitVuXfxTQazWAwyNq1a5XXMZyWkZEhc+fOHbH1TZkyRXw+n/z5z3+Ws2fPSjgcVv0nOGxdXV3y/vvvy2233SY6nW7Ej/1A5yjfjDoCrFYrli5dir/+9a945513sGLFClitVtVl0QhLTk7GI488gg8++ADvvfcefvzjH8PpdI6Lmc5wmUwmLF++HIWFhcjLy7vmBeNRMUZBO6LG2wzocuFwWNrb2+XDDz+UlStXSmxsrPIZwEi0yToDMhqNcvPNN8vmzZulrq5Ouru7o3rGcy1dXV2yb98+WbBggej1+jGZATGARlF/EN1zzz1R/9RssgWQ1WqVO+64Q958800JBAITNnSuFA6HJRAIyKZNm8ThcDCAriZaAkik7xfa0dEh+/btk3vvvTdqZ0STJYBcLpc8/PDDUlRUJG1tbZMmeK7U3d0txcXFcvvtt4vRaBwfAfT73/9e5syZIzabTWw2m2RlZUlhYaHW397eLuvXr5fExESZMmWK3H///eL3+yPWcebMGe1ircvlkqeeekq6u7uHUkZUBdDl2tvbpaioSH75y1/KbbfdJnFxcSM21WUADT+ADAaDfOc735Hf/OY3Ul1dLV1dXar/VMaFcDgsZ8+elc2bN4vL5VIfQO+++668//77cuLECamrq5Nf//rXYjKZpLq6WkRE1q1bJ2lpaVJUVCTl5eWSlZUlixYt0h7f09Mj6enp4vP5pLKyUgoLC8XpdEp+fv6QDky0BpBI3y81HA5LKBSSqqoq2bJli6xatUrS0tLEZDIpP1knUwDFxMTI4sWL5fXXX5fGxkbtd0ORenp65JNPPpG77rpryLOhUX8KlpCQIG+88Ya0tLSIyWSSnTt3an3Hjh0TAFJaWioiIoWFhaLX6yNmRdu2bRO73S6dnZ2D3mY0B9CVwuGwdHZ2yldffSXvv/++5OXlSWZmpjgcjnE1O5pIAZSYmCgPPvig7N27V0KhEENnEMLhsDQ3N8vzzz8vbrdbfQD19PTI22+/LWazWWpqaqSoqEgAyIULFyLGeb1eeemll0RE5JlnnpG5c+dG9H/xxRcCQA4dOjTobU+kALpc/7/Ara2tcvToUdm2bZv88Ic/lOnTpyufHUV7AM2bN09mzJghTz31lFRVVUlnZyeDZxh6enrkn//8p/h8PjEYDNcdQEN+JfTRo0eRnZ2Njo4OxMXFYffu3Zg9ezaqqqpgNpsRHx8fMd7tdsP/Px+P5/f74Xa7v9Hf3/dtOjs70XnZ53yGQqGhlh0V+l9PEhcXh/T0dNxyyy34+c9/jubmZhw5cgTFxcX45JNPUFtbi2AwiPAofHqW0WiEyWSCyWSC2WxGYmIikpKS4HQ64XK58JOf/GTEtznabDYbFi5ciLvuugupqanQ6/nyt+EyGAzIysrCzp078dprr+H111/Hv/71r2Gvb8gB9N3vfhdVVVUIBoPYtWsX1q5di5KSkmEXMBgFBQXYvHnzqG5jPNLpdDAajXC73Vi6dCl8Ph8uXryI+vp6lJaW4h//+AfKy8tx5swZdF/jk7V0Op3WrFar9gZGu92O5ORkeDwepKSkaJ+N1B82SUlJiImJgdlshtlshslkiuoX3dHI0Ol0iI+PR15eHpYuXYrnn38e77333rA+DuS6P47D5/PhhhtuwIMPPoglS5bgwoULEbOgadOmYePGjfjVr36FZ599Fu+++y6qqqq0/lOnTmHmzJk4dOgQ5s2bd9VtXG0GlJaWNuBb/ScyEUE4HEZzczNqa2tRXFyMkpISfP3113A6nUhOTobL5dI+eK0/YBISEhAXFwebzYa4uDjtXdn9wcKAoaEQEbS2tuKtt97Ciy++iIaGhoiZ+YDn6PU+J7zzzjtl7dq12kXoXbt2aX3Hjx8X4JsXoQOBgDbm1VdfFbvdLh0dHYPe5kS9BnQ9wuGwXLx4UVpaWqStrU06Ojqkt7eX1zloTPT09Mjhw4dl9erVEhcXNzoXoTdt2iQlJSVy6tQpOXLkiGzatEl0Op18+OGHItJ3G97r9UpxcbGUl5dLdna2ZGdnRxSZnp4uy5Ytk6qqKtm7d6+4XK5JdRueaKLqv4GydetWmTlz5sgH0COPPCLTpk0Ts9ksLpdLlixZooWPyL9fiJiQkCCxsbGyevVqaWxsjFjH6dOnZfny5WK1WsXpdEpeXt6keSEi0WTQ29srZWVlgzpHo/ojWSfzNSCi8Wyw5yjvRxKRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpAwDiIiUYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETKMICISBkGEBEpwwAiImUYQESkDAOIiJRhABGRMgwgIlKGAUREyjCAiEgZBhARKcMAIiJlGEBEpIxRdQHDISIAgFAopLgSIrqa/nOz/1z9NlEZQM3NzQCAtLQ0xZUQ0bW0trbC4XB8a39UBlBiYiIAoL6+/po7R5FCoRDS0tLQ0NAAu92uupyowGM2PCKC1tZWpKamXnNcVAaQXt936crhcPCPYhjsdjuP2xDxmA3dYCYHvAhNRMowgIhImagMIIvFgueeew4Wi0V1KVGFx23oeMxGl04Guk9GRDRKonIGREQTAwOIiJRhABGRMgwgIlImKgNo69atmD59OmJiYpCZmYmysjLVJSlTUFCABQsWwGazITk5GatWrUJdXV3EmI6ODuTm5iIpKQlxcXF44IEHEAgEIsbU19dj5cqViI2NRXJyMp5++mn09PSM5a4o88ILL0Cn02Hjxo3aMh6zMSJRZseOHWI2m+WPf/yj1NTUyKOPPirx8fESCARUl6ZETk6ObN++Xaqrq6WqqkpWrFghXq9X2tratDHr1q2TtLQ0KSoqkvLycsnKypJFixZp/T09PZKeni4+n08qKyulsLBQnE6n5Ofnq9ilMVVWVibTp0+XW2+9VZ544gltOY/Z2Ii6AFq4cKHk5uZqP/f29kpqaqoUFBQorGr8aGpqEgBSUlIiIiItLS1iMplk586d2phjx44JACktLRURkcLCQtHr9eL3+7Ux27ZtE7vdLp2dnWO7A2OotbVVbrzxRtm3b5/ccccdWgDxmI2dqHoK1tXVhYqKCvh8Pm2ZXq+Hz+dDaWmpwsrGj2AwCODfb9itqKhAd3d3xDGbNWsWvF6vdsxKS0sxZ84cuN1ubUxOTg5CoRBqamrGsPqxlZubi5UrV0YcG4DHbCxF1ZtRz507h97e3ohfOgC43W4cP35cUVXjRzgcxsaNG7F48WKkp6cDAPx+P8xmM+Lj4yPGut1u+P1+bczVjml/30S0Y8cOHDp0CAcPHvxGH4/Z2ImqAKJry83NRXV1NT799FPVpYxrDQ0NeOKJJ7Bv3z7ExMSoLmdSi6qnYE6nEwaD4Rt3IwKBADwej6KqxocNGzZgz549+OijjzB16lRtucfjQVdXF1paWiLGX37MPB7PVY9pf99EU1FRgaamJtx2220wGo0wGo0oKSnBli1bYDQa4Xa7eczGSFQFkNlsRkZGBoqKirRl4XAYRUVFyM7OVliZOiKCDRs2YPfu3SguLsaMGTMi+jMyMmAymSKOWV1dHerr67Vjlp2djaNHj6KpqUkbs2/fPtjtdsyePXtsdmQMLVmyBEePHkVVVZXW5s+fjzVr1mj/z2M2RlRfBR+qHTt2iMVikTfffFNqa2vlsccek/j4+Ii7EZPJ448/Lg6HQz7++GNpbGzU2qVLl7Qx69atE6/XK8XFxVJeXi7Z2dmSnZ2t9fffUl62bJlUVVXJ3r17xeVyTapbypffBRPhMRsrURdAIiKvvPKKeL1eMZvNsnDhQvnss89Ul6QMgKu27du3a2Pa29tl/fr1kpCQILGxsbJ69WppbGyMWM/p06dl+fLlYrVaxel0Sl5ennR3d4/x3qhzZQDxmI0NfhwHESkTVdeAiGhiYQARkTIMICJShgFERMowgIhIGQYQESnDACIiZRhARKQMA4iIlGEAEZEyDCAiUoYBRETK/H/+aFQQ1RQ0sQAAAABJRU5ErkJggg==\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "from matplotlib import pyplot as plt\n",
    "\n",
    "%matplotlib inline\n",
    "\n",
    "\n",
    "#打印游戏\n",
    "def show():\n",
    "    plt.figure(figsize=(3, 3))\n",
    "    plt.imshow(env.render())\n",
    "    plt.show()\n",
    "\n",
    "\n",
    "show()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "f489de60",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "env.observation_space= Box([-1.5       -1.5       -5.        -5.        -3.1415927 -5.\n",
      " -0.        -0.       ], [1.5       1.5       5.        5.        3.1415927 5.        1.\n",
      " 1.       ], (8,), float32)\n",
      "env.action_space= Discrete(4)\n",
      "state= [-1.2488365e-03  1.4045997e+00 -1.2650049e-01 -2.8090703e-01\n",
      "  1.4537866e-03  2.8654184e-02  0.0000000e+00  0.0000000e+00]\n",
      "action= 2\n",
      "next_state= [-0.00238953  1.3991038  -0.11603234 -0.24426208  0.00339713  0.03887122\n",
      "  0.          0.        ]\n",
      "reward= 3.8206632939835687\n",
      "done= False\n",
      "info= {}\n"
     ]
    }
   ],
   "source": [
    "#认识游戏环境\n",
    "def test_env():\n",
    "    print('env.observation_space=', env.observation_space)\n",
    "    print('env.action_space=', env.action_space)\n",
    "\n",
    "    state = env.reset()\n",
    "    action = env.action_space.sample()\n",
    "    next_state, reward, done, info = env.step(action)\n",
    "\n",
    "    print('state=', state)\n",
    "    print('action=', action)\n",
    "    print('next_state=', next_state)\n",
    "    print('reward=', reward)\n",
    "    print('done=', done)\n",
    "    print('info=', info)\n",
    "\n",
    "\n",
    "test_env()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "6d5c572d",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "<stable_baselines3.ppo.ppo.PPO at 0x7f1d1f0c77c0>"
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stable_baselines3 import PPO\n",
    "from stable_baselines3.common.env_util import make_vec_env\n",
    "\n",
    "#初始化模型\n",
    "model = PPO(\n",
    "    policy='MlpPolicy',\n",
    "    env=make_vec_env(MyWrapper, n_envs=8),  #使用N个环境同时训练\n",
    "    learning_rate=1e-3,\n",
    "    n_steps=1024,  #运行N步后执行更新,buffer_size=n_steps*环境数量\n",
    "    batch_size=64,  #采样数据量\n",
    "    n_epochs=16,  #每次采样后训练的次数\n",
    "    gamma=0.99,\n",
    "    verbose=0)\n",
    "\n",
    "model"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "22afd2bc",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/root/anaconda3/envs/pt39/lib/python3.9/site-packages/stable_baselines3/common/evaluation.py:67: UserWarning: Evaluation environment is not wrapped with a ``Monitor`` wrapper. This may result in reporting modified episode lengths and rewards, if other wrappers happen to modify these. Consider wrapping environment first with ``Monitor`` wrapper.\n",
      "  warnings.warn(\n"
     ]
    },
    {
     "data": {
      "text/plain": [
       "(-186.2003113092622, 127.51220384337816)"
      ]
     },
     "execution_count": 5,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from stable_baselines3.common.evaluation import evaluate_policy\n",
    "\n",
    "evaluate_policy(model, env, n_eval_episodes=20, deterministic=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "75e0b593",
   "metadata": {
    "scrolled": false
   },
   "outputs": [],
   "source": [
    "# #训练\n",
    "# model.learn(100_0000, progress_bar=True)\n",
    "\n",
    "# #保存模型\n",
    "# model.save('save/1.PPO.Lunar Lander')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "id": "cdaaf12a",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(265.7510707758057, 57.85305098127015)"
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "#加载模型\n",
    "model = PPO.load('save/1.PPO.Lunar Lander')\n",
    "\n",
    "evaluate_policy(model, env, n_eval_episodes=20, deterministic=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "id": "414e1478",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "image/png": "iVBORw0KGgoAAAANSUhEUgAAASAAAADMCAYAAADTcn7NAAAAOXRFWHRTb2Z0d2FyZQBNYXRwbG90bGliIHZlcnNpb24zLjYuMywgaHR0cHM6Ly9tYXRwbG90bGliLm9yZy/P9b71AAAACXBIWXMAAA9hAAAPYQGoP6dpAAAZ1klEQVR4nO3de1CU1/0/8PezV667BNBdEfBS/QYVYxUVV/P7pZmg1NBEE5PJOLY6xsaRYCZWm2mZac2k85viJDNNm9aaznynNjPfSe2YGdv+rNHyQ8XaUC8IykXRhiIk7EVuu8tlF3b3/P6gPHUNKiDsYfX9mjkDPOewz2cf2DfPc87uogghBIiIJNDILoCIHl0MICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikkZaAO3fvx8zZ85ETEwMcnNzcf78eVmlEJEkUgLoD3/4A3bv3o23334bly5dwqJFi5Cfnw+XyyWjHCKSRJHxYtTc3FwsW7YMv/rVrwAAoVAIGRkZeOONN/DDH/4w0uUQkSS6SO+wv78flZWVKC4uVrdpNBrk5eWhoqJi2O/x+/3w+/3q16FQCB0dHUhJSYGiKBNeMxGNjhACXq8XaWlp0GjufqEV8QBqa2tDMBiExWIJ226xWHDt2rVhv6ekpATvvPNOJMojonHU0tKC9PT0u/ZHxSpYcXEx3G632pqbm2WXREQjkJiYeM/+iJ8BpaamQqvVwul0hm13Op2wWq3Dfo/RaITRaIxEeUQ0ju43RRLxMyCDwYCcnByUlZWp20KhEMrKymCz2SJdDhFJFPEzIADYvXs3tmzZgqVLl2L58uX4+c9/jp6eHmzdulVGOUQkiZQAeuWVV3Dr1i3s3bsXDocDX//613H8+PGvTEwT0cNNyvOAHpTH44HZbJZdBhHdh9vthslkumt/VKyCEdHDiQFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUkz6gA6c+YMnnvuOaSlpUFRFPzxj38M6xdCYO/evZg2bRpiY2ORl5eHGzduhI3p6OjApk2bYDKZkJSUhG3btqG7u/uB7ggRRZ9RB1BPTw8WLVqE/fv3D9v/7rvv4oMPPsCHH36Ic+fOIT4+Hvn5+fD5fOqYTZs2oa6uDqWlpTh69CjOnDmD7du3j/1eEFF0Eg8AgDhy5Ij6dSgUElarVbz33nvqtq6uLmE0GsXvf/97IYQQ9fX1AoC4cOGCOubTTz8ViqKIL7/8ckT7dbvdAgAbG9skb263+56P5XGdA/rXv/4Fh8OBvLw8dZvZbEZubi4qKioAABUVFUhKSsLSpUvVMXl5edBoNDh37tywt+v3++HxeMIaEUW/cQ0gh8MBALBYLGHbLRaL2udwODB16tSwfp1Oh+TkZHXMnUpKSmA2m9WWkZExnmUTkSRRsQpWXFwMt9uttpaWFtklEdE4GNcAslqtAACn0xm23el0qn1WqxUulyusPxAIoKOjQx1zJ6PRCJPJFNaIKPqNawDNmjULVqsVZWVl6jaPx4Nz587BZrMBAGw2G7q6ulBZWamOOXnyJEKhEHJzc8ezHCKa7Eax6CWEEMLr9YqqqipRVVUlAIif/exnoqqqSty8eVMIIcS+fftEUlKS+NOf/iSuXLki1q1bJ2bNmiX6+vrU2/jmN78pFi9eLM6dOyfOnj0r5s6dKzZu3DjiGrgKxsYWHe1+q2CjDqBTp04Nu6MtW7YIIQaX4n/84x8Li8UijEajeOaZZ0RDQ0PYbbS3t4uNGzeKhIQEYTKZxNatW4XX62UAsbE9ZO1+AaQIIQSijMfjgdlsll0GEd2H2+2+55xtVKyCEdHDiQFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUkzqgAqKSnBsmXLkJiYiKlTp2L9+vVoaGgIG+Pz+VBUVISUlBQkJCRgw4YNcDqdYWOam5tRUFCAuLg4TJ06FW+99RYCgcCD3xsiii5iFPLz88XBgwdFbW2tqK6uFs8++6zIzMwU3d3d6pgdO3aIjIwMUVZWJi5evChWrFghVq5cqfYHAgGRnZ0t8vLyRFVVlTh27JhITU0VxcXFI67D7XYLAGxsbJO8ud3uez6WRxVAd3K5XAKAKC8vF0II0dXVJfR6vTh8+LA65urVqwKAqKioEEIIcezYMaHRaITD4VDHHDhwQJhMJuH3+0e0XwYQG1t0tPsF0APNAbndbgBAcnIyAKCyshIDAwPIy8tTx2RlZSEzMxMVFRUAgIqKCixcuBAWi0Udk5+fD4/Hg7q6umH34/f74fF4whoRRb8xB1AoFMKuXbuwatUqZGdnAwAcDgcMBgOSkpLCxlosFjgcDnXM7eEz1D/UN5ySkhKYzWa1ZWRkjLVsIppExhxARUVFqK2txaFDh8aznmEVFxfD7XarraWlZcL3SUQTTzeWb9q5cyeOHj2KM2fOID09Xd1utVrR39+Prq6usLMgp9MJq9Wqjjl//nzY7Q2tkg2NuZPRaITRaBxLqUQ0mY1m0jkUComioiKRlpYmrl+//pX+oUnoTz75RN127do1AXx1EtrpdKpjfvOb3wiTySR8Pt+I6uAkNBtbdLRxXQUrLCwUZrNZnD59WtjtdrX19vaqY3bs2CEyMzPFyZMnxcWLF4XNZhM2m03tH1qGX7NmjaiurhbHjx8XU6ZM4TI8G9tD2MY1gO62k4MHD6pj+vr6xOuvvy4ee+wxERcXJ1544QVht9vDbqepqUmsXbtWxMbGitTUVLFnzx4xMDDAAGJje8ja/QJI+XewRBWPxwOz2Sy7DCK6D7fbDZPJdNd+vhaMiKRhABGRNAwgIpKGAURE0jCAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKSZkzvB0SPHkUBXnwR0GqBqirA7Qba2oBQSHZl92exAM89B3z++WDr7AS8XtlVEcAAohFSFGDdOiArCxgYADweoKkJ+OKLwUByOAYf3D4f4PfLrjbclCnAa68BQgzW3toK2O1AbS3Q2DjY2tqAvj4gGJRd7aOFAUSjotEARuPgg3rKFGDZMmD9+sHg6ewErl4F3n9/MJAmE0UZPHvT6YCvfW2wrVo1GEoeD9DRARw9CvzP/0THWd3DggFEozZ0JuHxAC4XUF8PNDcDly8PPpDv+D+Uk4YQg623d/AS7MaNwbqvXBk8k7PbGT6RxgCiEevosOLYMQcuXhwMmevXB898+vpkV3ZvGk0iamoEqqq60dAA3Lw5GDY+32CQkjwMIBoRRdGiqiofH330kexSRk2v/y988EEAly9fll0K3YEBRBMqMcGC7KxvQaPoAChQwnqVsE+DoX7UXvu/8Ha7IlskScMAogn1xH+tx6qFOxGjG3oLXeWuY/0BDzo6m9DQXRaZ4ki6qA6gJUuWoLW1FV6vFz6fD0GuoU4yCvT6WBi08f8+A7o3gzYeMTF8r+9HSVQH0J///GcoioLm5mY0Njbi+vXrqK+vx+effw6n04m2tjYMDAwgxKUNKWKMJmROWw6tYhjReEXRYvb0/4Ur9X+EEJP/Z2Y0GvHkk0/i+eefhxACPp8PfX198Pl89/y8v78fAwMDGBgYQCAQuO/HUCgEMfgfbBCF/0PinqI6gBITE2EymZCWlobc3FwAgBACfr8f7e3tcDqduHHjBq5evYobN27gn//8J7744gt4vV709PQ8dD/MyUdAp4vBvS67bqdAA70uFoqimdQBFB8fj2984xsoKirCU089hdjY2GHH3e33KxQKIRAIjKj19/ejra0NDocDdrsdDocDt27dgsvlgsvlQnt7uxpsfr8fA1G2rBfVAXQ7RVHUj7GxsUhPT0d6ejpycnIghFB/mEP/W76pqQlXr15FfX09mpqa8OWXX6KjowMDAwO8lBsnlpR5iDMmj3i8oihINc+FKcGKLs8XE1jZ2CQmJmL16tUoLCzEqlWrEBMTo/7eDedufRqNBjrd2B56QggEg0E1oPx+Pzo7O9HR0YGOjg60tbXBbrfDbrejtbUVLpcLXV1dauvr60MwGEQoFJoUv+cPTQDdi6Io0Ov10Ov1iI+PR1paGpYvX6729/b2oqurC06nE9evX0dtbS3Ky8tRV1eHzs5OiZVHt5iYRMToRzenE2dIht5gnKCKxsZsNqOgoACFhYVYtmwZDAbDPYNnIimKAp1OpwZYQkICUlJS1P47z7qCwSD6+vrQ29uLvr4+9ffc6XTC4XDgxo0buHz5MlpaWtDe3h7xM6hHIoCGc/svUHx8POLj4zF9+nQsWbJEvZ5vbGzE2bNnceLECVy4cAFOpzPqTnEjTas1ICEhFd3dtxAI+aBV9KN6sGo1ehiNJmi1eiQkTEFvbwcGBnwTWPHdJScnY/369di+fTsWL14MvX5090WGO+vT6XRITExEYmLiV8YKIRAKhdDf3w+Xy4XGxkbU1NSgsrIS165dQ2NjI7xeL/r7+ydsuuKRDaB7GbqMW7BgARYsWIBXX30VTqcTFy5cQGlpKcrLy9HY2AifT84DYzKbMX0Z/vfSN1HfdBSdnc1QlNG+48vgPNA3bLswa9qTOHb2R2i110xIrXeTmpqKl19+Ga+99hqys7Oh1+sjuv9IURQFWq0WsbGxmDFjBmbMmIGnn34aoVAIPT096OjoQENDA2pra1FVVYXa2lp1qmK8Lt8YQCOg1+uRnp6O6dOnY926dfB4PKirq0NZWRnKyspQU1MDt9v9yK+2aTQ6PPH4i7Cas/GZ+8Mx/9UMBvuBoIIpiY9jbubTaLXXYvBfjU8cRVEwbdo0vPLKK/jud7+Lxx9/HBqNZtKf8UwEjUajnjXNmDEDq1evRigUgs/ng91uR2NjI6qqqnD58mXU19ejpaUF3d3d6O/vH/W+RhVABw4cwIEDB9DU1AQAWLBgAfbu3Yu1a9cCAHw+H/bs2YNDhw7B7/cjPz8fv/71r2GxWNTbaG5uRmFhIU6dOoWEhARs2bIFJSUlY56UiyRFUaAoCpKSkrBq1SqsXLkS3//+93Hz5k387W9/Uy/V7HY7AoGA7HIj7jFzJqalLERnz0186bgC65R58AXcCImR/7XsD3gREkHcuHkK2V97HnPSn8b5Kx+hr889ITUrioKMjAxs2rQJW7duxezZs6HVaidkX9Fq6EwpPj4ec+bMwZw5c7B69WoEg0H09vbC6XSioaEBNTU1uHTpEhoaGtDc3Ay3+/4/s1E96tPT07Fv3z7MnTsXQgh89NFHWLduHaqqqrBgwQJ873vfw1/+8hccPnwYZrMZO3fuxIsvvoi///3vAAYnxAoKCmC1WvHZZ5/Bbrdj8+bN0Ov1+OlPfzq2oyORoiiIi4vDvHnzkJWVhW3btsHhcKCyshJ//etf1Uu13t5e2aVGxPw5z8Icl4GL1z9Cb28H3B47/nHlv7/yAoxw4X0CQXg9TvT53Ghtv4LZ1qcwzZKNxqa/j2utiqJg9uzZ2Lx5MzZv3ozMzExoNHyD0JEamgw3mUwwmUyYM2cOCgoKIISA1+tFY2MjlixZcv/bEQ84u5ScnIz33nsPL730EqZMmYKPP/4YL730EgDg2rVrmDdvHioqKrBixQp8+umn+Na3voXW1lb1rOjDDz/ED37wA9y6dQsGw8iesObxeGA2m+F2u2EymR6k/AkTDAbh9XpRX1+PkydPoqysDFeuXFEv1RRFUU/xb/+o0WgQExODmJgYxMbGqh9v/3zo9DgxMREJCQlhnwcCATgcDrS2tsJut+PWrVvo7OxUm8/nQyAQQDAYHNV1vFarxbe//e27vhg1LuYxvLD6faSnLsUf/t82NLWce+BjuGjei/jmip/gcuNhnDjzfyBGcSZ1u5ycHAQCgy9G1Wq1mDt3Ll599VVs3LgRaWlpDJ4JMNLH6Jive4LBIA4fPoyenh7YbDZUVlZiYGAAeXl56pisrCxkZmaqAVRRUYGFCxeGXZLl5+ejsLAQdXV1WLx48VjLmXS0Wi2SkpKwcuVK2Gw27NmzBzdv3sRnn30Gj8ejrrzFx8cjISEh7Gu9Xq8utep0urCvR3t5EAgE1Gfi3r4M63K5wpZjHQ4HnE6n+iTNnp4e9PX1jXhea+Z0G0yJaXB01cBx6+pYDtlXNH1xDu3d/4Q1dT7i4pLQ09M+5tvSaDTIzs7G9u3b8fLLL8NisTyS8zuTzagDqKamBjabDT6fDwkJCThy5Ajmz5+P6upqGAwGJCUlhY23WCxw/Pvt8RwOR1j4DPUP9d2N3++H/7b3+fR4PKMtW6qhVbWsrCxkZWVFdN9Dz38aWoadMWOG2nf70/uHnpjW3d0Nj8cDt9uthlVrayscDgc0Gg02b9487H4S462Yld6L/lAQr5heHJdnMiuKFrO/5kVvbxdeTnweodDYzoASExPx5JNPIi8vDykpKQyeSWTUAfT444+juroabrcbn3zyCbZs2YLy8vKJqE1VUlKCd955Z0L38SgamlQHBs/Y9Ho9YmJikJqa+oC3vOnBi6NHwqgvfg0GA+bMmYOcnByUlJRg0aJF+MUvfgGr1Yr+/n50dXWFjXc6nbBarQAAq9UK5x3v1zn09dCY4RQXF8PtdqutpaVltGUT0ST0wLNvoVAIfr8fOTk50Ov1KCv7z3u5DC3H2Ww2AIDNZkNNTQ1crv+84VRpaSlMJhPmz59/130YjUZ1tn2oEVH0G9UlWHFxMdauXYvMzEx4vV58/PHHOH36NE6cOAGz2Yxt27Zh9+7dSE5OhslkwhtvvAGbzYYVK1YAANasWYP58+fjO9/5Dt599104HA786Ec/QlFREYzGyfX6HyKaeKMKIJfLhc2bN8Nut8NsNuOJJ57AiRMnsHr1agDA+++/D41Ggw0bNoQ9EXGIVqvF0aNHUVhYCJvNhvj4eGzZsgU/+clPxvdeEVFUeODnAckQDc8DInqUjfQxymdgEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNAwgIpKGAURE0jCAiEgaBhARScMAIiJpGEBEJA0DiIikYQARkTQMICKShgFERNIwgIhIGgYQEUnDACIiaRhARCQNA4iIpGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkjU52AWMhhAAAeDweyZUQ0XCGHptDj9W7icoAam9vBwBkZGRIroSI7sXr9cJsNt+1PyoDKDk5GQDQ3Nx8zztH4TweDzIyMtDS0gKTySS7nKjAYzY2Qgh4vV6kpaXdc1xUBpBGMzh1ZTab+UsxBiaTicdtlHjMRm8kJwechCYiaRhARCRNVAaQ0WjE22+/DaPRKLuUqMLjNno8ZhNLEfdbJyMimiBReQZERA8HBhARScMAIiJpGEBEJE1UBtD+/fsxc+ZMxMTEIDc3F+fPn5ddkjQlJSVYtmwZEhMTMXXqVKxfvx4NDQ1hY3w+H4qKipCSkoKEhARs2LABTqczbExzczMKCgoQFxeHqVOn4q233kIgEIjkXZFm3759UBQFu3btUrfxmEWIiDKHDh0SBoNB/Pa3vxV1dXXitddeE0lJScLpdMouTYr8/Hxx8OBBUVtbK6qrq8Wzzz4rMjMzRXd3tzpmx44dIiMjQ5SVlYmLFy+KFStWiJUrV6r9gUBAZGdni7y8PFFVVSWOHTsmUlNTRXFxsYy7FFHnz58XM2fOFE888YR488031e08ZpERdQG0fPlyUVRUpH4dDAZFWlqaKCkpkVjV5OFyuQQAUV5eLoQQoqurS+j1enH48GF1zNWrVwUAUVFRIYQQ4tixY0Kj0QiHw6GOOXDggDCZTMLv90f2DkSQ1+sVc+fOFaWlpeKpp55SA4jHLHKi6hKsv78flZWVyMvLU7dpNBrk5eWhoqJCYmWTh9vtBvCfF+xWVlZiYGAg7JhlZWUhMzNTPWYVFRVYuHAhLBaLOiY/Px8ejwd1dXURrD6yioqKUFBQEHZsAB6zSIqqF6O2tbUhGAyG/dABwGKx4Nq1a5KqmjxCoRB27dqFVatWITs7GwDgcDhgMBiQlJQUNtZiscDhcKhjhjumQ30Po0OHDuHSpUu4cOHCV/p4zCInqgKI7q2oqAi1tbU4e/as7FImtZaWFrz55psoLS1FTEyM7HIeaVF1CZaamgqtVvuV1Qin0wmr1Sqpqslh586dOHr0KE6dOoX09HR1u9VqRX9/P7q6usLG337MrFbrsMd0qO9hU1lZCZfLhSVLlkCn00Gn06G8vBwffPABdDodLBYLj1mERFUAGQwG5OTkoKysTN0WCoVQVlYGm80msTJ5hBDYuXMnjhw5gpMnT2LWrFlh/Tk5OdDr9WHHrKGhAc3Nzeoxs9lsqKmpgcvlUseUlpbCZDJh/vz5kbkjEfTMM8+gpqYG1dXValu6dCk2bdqkfs5jFiGyZ8FH69ChQ8JoNIrf/e53or6+Xmzfvl0kJSWFrUY8SgoLC4XZbBanT58Wdrtdbb29veqYHTt2iMzMTHHy5Elx8eJFYbPZhM1mU/uHlpTXrFkjqqurxfHjx8WUKVMeqSXl21fBhOAxi5SoCyAhhPjlL38pMjMzhcFgEMuXLxf/+Mc/ZJckDYBh28GDB9UxfX194vXXXxePPfaYiIuLEy+88IKw2+1ht9PU1CTWrl0rYmNjRWpqqtizZ48YGBiI8L2R584A4jGLDL4dBxFJE1VzQET0cGEAEZE0DCAikoYBRETSMICISBoGEBFJwwAiImkYQEQkDQOIiKRhABGRNAwgIpKGAURE0vx/IeKAFOlX6BEAAAAASUVORK5CYII=\n",
      "text/plain": [
       "<Figure size 300x300 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "269.0743261571944 162 [1.9535648857281274, 2.1027781425759997, 2.16199896559138, 2.215160996075781, 2.2650470259618203, 2.3085327551518446, 2.3388834755316736, 2.3377030355311774, 2.9992252764444074, 1.5155375724942541, 0.17756842505556847, -1.7851174240369687, -2.287398042555168, -2.36924663018371, -2.362138367441588, -2.4510520838217063, -2.4281164788774263, -2.2443180358797576, -2.1892792346012584, -2.132731770360067, -2.2736083142014523, -2.1096894770403183, -1.933892398327231, -1.7188953089604115, -1.6685296110653667, -1.6460600414626458, -1.6465209810120314, -1.5871558499736977, -1.5276759086796972, -1.46813210459851, -1.4085233447681276, -1.348845255755208, -1.2891413391053277, -1.229403711692271, -1.1696212917637467, -1.2487630542336763, -1.4463144353769326, -1.1479506965648614, -1.0505088542637395, -0.9906119617407967, -0.8196449260621523, -0.6663029947910388, -0.8377871821362521, -0.6347848869889663, -0.6364621743526289, -0.7955001359614176, -0.6108429692938262, -0.4651523044323358, -0.11949423814124202, -0.30748266773932076, 4.7221117512281525, 3.343533936401127, 3.0375517732196444, 5.194146175326819, 3.8714931365165173, 2.281970793850758, -0.7864146153130196, -0.5484912234064143, 4.313448323844125, 2.30328816663386, 2.339237460233801, 5.869743405102651, 5.348053010807303, 1.85204942770211, 5.938311183118185, 4.9852208667519, -0.6058182584655174, -0.6814556993397491, 5.102516934357721, -0.9154147444334331, -0.6200715367944827, 2.138068207584735, 2.648473936402735, 2.7963182060962994, 5.364931335321541, 3.737501698731177, 4.963359782570746, 4.409543775150209, 3.3334390392689928, 2.1689857463400815, -1.1456017163549024, 4.598787650774281, -1.2396869740730228, 5.072390804971431, 4.81397212844895, 0.6422766106900297, -1.3668173276048055, 2.0000662407790175, 5.06324963464125, 3.4639826947348924, -1.133818916679586, -1.159833313860729, 3.482966790871015, 3.4726767608606197, 3.8369247716488077, 2.461825468047448, 1.7551453463108373, 1.1207962043710438, 4.418767213348265, 4.038021887073744, 2.96079554315848, 1.3504038025295386, 0.6768632770831828, 3.3927250333539236, 2.4450108311127936, 2.4358189403170813, 1.522127403486192, 0.753230242464366, 2.1604250209848077, 1.149194532839229, 0.39709606267750247, 22.29565851391273, 7.014279336055638, 7.1233067486283845, 6.9863814983468915, 2.687336144304682, 2.5133849647723974, 1.8634490950774918, -1.4426047957317216, -1.7765716715996653, -1.4783123686763862, -1.0125620870053762, -0.4931582491438906, 0.013213498311156258, 0.45139725410943043, 0.771983456639477, 0.9400521158821711, 0.9453135097782237, 0.8058090755806813, 0.5640325131695381, 0.276906627739713, -0.0020671333393789126, -0.2367073013659482, -0.4019136934711369, 0.5940129318847926, 1.8023354984518623, 0.3868888077776589, 0.15057636140861774, -0.04390799056717043, 0.058102521028917664, 0.0472524614705776, -0.01571431660007505, 0.006686327701295625, 0.009437204843003144, -0.0014010897277536571, 0.0005986650246967429, 0.0016339487821035448, 0.0001502082014148698, -4.7094609882947225e-05, 0.0002531660080258291, 0.00010699293698124279, -3.3708413404553994e-05, 3.759739799491513e-05, 3.258722678367576e-05, -1.0757537634020764e-05, 2.987944384358343e-06, 5.2983901213110585e-06, 3.100798402044802e-06, -1.924401487940486e-06, -1.1040235392556497e-06, 1.7121091389071807e-06, 100]\n"
     ]
    }
   ],
   "source": [
    "from IPython import display\n",
    "import random\n",
    "\n",
    "\n",
    "def test():\n",
    "    state = env.reset()\n",
    "    reward_sum = []\n",
    "    over = False\n",
    "    while not over:\n",
    "        action, _ = model.predict(state)\n",
    "        state, reward, over, _ = env.step(action)\n",
    "        reward_sum.append(reward)\n",
    "\n",
    "        if len(reward_sum) % 5 == 0:\n",
    "            display.clear_output(wait=True)\n",
    "            show()\n",
    "\n",
    "    print(sum(reward_sum), len(reward_sum), reward_sum)\n",
    "\n",
    "\n",
    "test()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.9.15"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
